"""
Case Type   : 故障&可靠性测试
Case Name   : 事务提交方式为remote_apply,主节点注入磁盘满故障
Create At   : 2021/03/01
Owner       : opengauss
Description :
    1.创建表，并插入数据
    2.设置synchronous_commit=remote_apply,
    synchronous_standby_names=dn_6002,dn_6003
    3.使用gs_guc set方式设置recovery_min_apply_delay
    4.重启数据库
    5.查询集群同步方式
    5.等待主备一致
    6.插入数据
    7.主节点注入磁盘满
    8.等待5s查询备
    9.清除故障
    10.查询备节点
    11.插入数据
    12.插入数据同时主节点注入磁盘满
    13.等待5s清除故障
    14.重启集群
    15.查询备节点
Expect      :
    1.设置成功
    2.设置成功
    3.重启成功
    4.集群为同步
    5.主备一致
    6.connection to server was lost
    7.故障注入成功
    8.未同步
    9.清除故障成功
    10.数据插入成功
    11.connection to server was lost
    12.注入故障成功
    13.清除故障成功
    14.重启集群成功
    15.主备同步
History     :
    Modified by @wan005 2021/3/17: 重启集群后数据库处于恢复过程中不一致状态增加延迟规避该问题
    Modified by @wan005 2021/5/17: 增加等待时间，规避无法获取线程结果问题并增加断言信息
    Modified by @wan005 2021/7/29: 修改用例先注入故障再insert 并优化断言
    Modified by ningyali 2024/7/9 优化用例解决一致性问题
"""

import os
import time
import unittest

from yat.test import Node
from yat.test import macro

from testcase.utils.ComThread import ComThread
from testcase.utils.Common import Common
from testcase.utils.CommonSH import CommonSH
from testcase.utils.CommonSH import RestartDbCluster
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger


class RecoveryDelay(unittest.TestCase):
    nodes_tuple = ('PrimaryDbUser', 'Standby1DbUser', 'Standby2DbUser')

    @RestartDbCluster(*nodes_tuple)
    def setUp(self):
        self.log = Logger()
        self.log.info(f'----{os.path.basename(__file__)} 开始执行------')
        self.pri_sh = CommonSH('PrimaryDbUser')
        self.pri_root = Node(node='PrimaryRoot')
        self.pri_user = self.pri_sh.node
        self.compri = Common('PrimaryRoot')
        self.comsta = [Common('Standby1Root'), Common('Standby2Root')]
        self.db_standby_node =  \
            [Node(node='Standby1Root'), Node(node='Standby2Root')]
        self.comshsta = \
            [CommonSH('Standby1DbUser'), CommonSH('Standby2DbUser')]
        self.constant = Constant()
        self.db_disk_name = self.compri.get_disk_name(macro.DB_INSTANCE_PATH)
        self.tb_name = 'tb_241'
        self.conf_path = os.path.join(
            macro.DB_INSTANCE_PATH, macro.DB_PG_CONFIG_NAME)

        self.log.info('------同步集群时间--------')
        current = self.pri_root.sh(
            "date \"+%m/%d/%Y %H:%M:%S\"").result()
        self.log.info(current)
        datecmd = f'date -s "{current}"'
        self.log.info(datecmd)
        for i in range(2):
            result = self.db_standby_node[i].sh(datecmd).result()
            self.log.info(result)

    def test_recoverydelay(self):
        self.log.info('--------创建表，并插入数据-------')
        sql = f"drop table if exists {self.tb_name};" \
            f"create table {self.tb_name}(i int, s char(10));"
        result = self.pri_sh.executDbSql(sql)
        self.log.info(result)
        self.assertIn(self.constant.TABLE_CREATE_SUCCESS, result)

        self.log.info('------备份postgres.conf文件----------')
        shell_cmd = f"cp {self.conf_path} {self.conf_path}_testbak"
        self.log.info(shell_cmd)
        result = self.pri_user.sh(shell_cmd).result()
        self.log.info(result)

        self.log.info('--------设置synchronous_commit=remote_apply-------')
        result = self.pri_sh.executeGsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            'synchronous_commit=remote_apply')
        self.assertTrue(result)

        self.log.info('---设置synchronous_standby_names=dn_6002,dn_6003--')
        shell_cmd = f"cat {self.conf_path} | " \
            f"grep synchronous_standby_names"
        result = self.pri_user.sh(shell_cmd).result()
        self.log.info(result)
        shell_cmd = f"sed -i \"s/" \
            f"{result.split('#')[0]}/synchronous_standby_names='" \
            f"{macro.DN_NODE_NAME.split('/')[1]}," \
            f"{macro.DN_NODE_NAME.split('/')[2]}'/g\" {self.conf_path}"
        self.log.info(shell_cmd)
        result_tmp = self.pri_user.sh(shell_cmd).result()
        self.log.info(result_tmp)

        self.log.info('-----设置recovery_min_apply_delay=3min----')
        result = self.pri_sh.executeGsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            'recovery_min_apply_delay=3min')
        self.assertTrue(result)

        self.log.info('----------重启数据库-----------')
        result = self.pri_sh.stopDbCluster()
        self.assertTrue(result)
        result = self.pri_sh.startDbCluster()
        self.assertTrue(result)

        self.log.info('-----------查询参数-----------')
        result = self.pri_sh.executDbSql('show synchronous_commit;')
        self.log.info(result)
        self.assertIn('remote_apply', result)
        result = self.pri_sh.executDbSql(
            'show recovery_min_apply_delay;')
        self.log.info(result)
        self.assertIn('3min', result)
        result = self.pri_sh.executDbSql(
            'show synchronous_standby_names;')
        self.log.info(result)
        self.assertIn(f"{macro.DN_NODE_NAME.split('/')[1]},"
                      f"{macro.DN_NODE_NAME.split('/')[2]}", result)

        self.log.info('--------查询集群同步方式-----')
        sql = "select * from pg_stat_replication;"
        result = self.pri_sh.executDbSql(sql)
        self.log.info(result)
        self.assertIn('Sync', result)
        self.assertIn('Potential', result)

        self.log.info('--------等待主备一致------------')
        for i in range(2):
            result = self.comshsta[i].check_data_consistency()
            self.assertTrue(result)

        self.log.info('-------------主节点注入磁盘满---------------')
        cfedisk_cmd = f'{macro.CFE_PATH}/cfe ' \
            f'"inject  rfile_full (diskname) values  ({self.db_disk_name})"'
        self.log.info(cfedisk_cmd)
        result = self.pri_root.sh(cfedisk_cmd).result()
        self.log.info(result)

        self.log.info('--------插入数据-------')
        sql = f"insert into {self.tb_name} values(1,'test');"
        insert_thread = ComThread(self.pri_sh.executDbSql, args=(sql,))
        insert_thread.setDaemon(True)
        insert_thread.start()
        self.log.info(result)

        self.log.info('========获取insert结果======')
        insert_thread.join(1800)
        result = insert_thread.get_result()
        self.log.info(result)
        flg = self.constant.CONNECT_LOST_MSG in result or \
              self.constant.NO_SPACE_BUILD_FAIL_MSG in result
        self.assertTrue(flg)

        self.log.info('----------查询备---------------')
        sql = f"select sysdate;select * from {self.tb_name};"
        result = self.pri_sh.executDbSql(sql)
        self.log.info(result)
        flg = self.constant.FAILED_CONNECT_DB in result or \
              self.constant.DISK_FULL_MSG in result
        self.assertTrue(flg)
        for i in range(2):
            result = self.comshsta[1].executDbSql(sql)
            self.log.info(result)
            self.assertIn('0 rows', result)

        self.log.info('----------------------清除故障----------------------')
        cfedisk_cmd = f'''{macro.CFE_PATH}/cfe "clean  
                rfile_full  where(diskname={self.db_disk_name})"'''
        tmp = self.pri_root.sh(cfedisk_cmd).result()
        self.log.info(tmp)

        time.sleep(5)
        self.log.info('----------重启数据库-----------')
        result = self.pri_sh.stopDbCluster()
        self.log.info(result)
        result = self.pri_sh.startDbCluster()
        self.assertTrue(result)

        time.sleep(180)
        result_pri = self.pri_sh.executDbSql(sql)
        self.log.info(result_pri)
        for i in range(2):
            result = self.comshsta[1].executDbSql(sql)
            self.log.info(result)
            if '1 | test' in result_pri:
                self.assertIn('1 | test', result)
            else:
                self.assertNotIn('1 | test', result)

        self.log.info('---------------插入数据------------------')
        sql = f"insert into {self.tb_name} values(3,'test');"
        insert_thread = ComThread(self.pri_sh.executDbSql, args=(sql,))
        insert_thread.setDaemon(True)
        insert_thread.start()

        self.log.info('-------------主节点注入故障---------------')
        cfedisk_cmd = f'{macro.CFE_PATH}/cfe ' \
            f'"inject  rfile_full (diskname) ' \
            f'values  ({self.db_disk_name})"'
        self.log.info(cfedisk_cmd)
        result = self.pri_root.sh(cfedisk_cmd).result()
        self.log.info(result)

        self.log.info('--------------等待5s清除故障----------------------')
        time.sleep(5)
        cfedisk_cmd = f'{macro.CFE_PATH}/cfe "clean rfile_full ' \
            f'where(diskname={self.db_disk_name})"'
        tmp = self.pri_root.sh(cfedisk_cmd).result()
        self.log.info(tmp)

        self.log.info('----------重启数据库-----------')
        time.sleep(5)
        result = self.pri_sh.stopDbCluster()
        self.log.info(result)
        result_start = self.pri_sh.startDbCluster(True)
        flg = 'Degraded' in result_start \
              or self.constant.START_SUCCESS_MSG in result_start
        self.assertTrue(flg)

        self.log.info('========获取insert结果======')
        insert_thread.join(80)
        result = insert_thread.get_result()
        self.log.info(result)
        flg = self.constant.CONNECT_LOST_MSG in result or \
              self.constant.DISK_FULL_MSG in result
        self.assertTrue(flg)

        self.log.info('----------查询备节点-------------------')
        sql = f"select sysdate;select * from {self.tb_name};"
        result_pri = self.pri_sh.executDbSql(sql)
        self.log.info(result)
        time.sleep(180)
        for i in range(2):
            result = self.comshsta[i].executDbSql(sql)
            self.log.info(result)
            if '3 | test' in result_pri:
                self.assertIn('3 | test', result)
            else:
                self.assertNotIn('3 | test', result)

    def tearDown(self):
        self.log.info('-----------------------------环境清理--------------------')
        self.log.info('----------------------清除故障---------------------------')
        cfedisk_cmd = f'''{macro.CFE_PATH}/cfe "clean 
        rfile_full  where(diskname={self.db_disk_name})"'''
        tmp = self.pri_root.sh(cfedisk_cmd).result()
        self.log.info(tmp)

        self.log.info('--------------还原配置文件-------------')
        shell_cmd = f"rm -rf {self.conf_path};" \
            f"cp {self.conf_path}_testbak {self.conf_path};" \
            f"rm -rf {self.conf_path}_testbak"
        self.log.info(shell_cmd)
        result = self.pri_user.sh(shell_cmd).result()
        self.log.info(result)

        self.log.info('-------------还原recovery_min_apply_delay----------')
        result = self.pri_sh.executeGsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            'recovery_min_apply_delay=0')
        self.log.info(result)

        self.log.info('-----------重启数据库-----------')
        result = self.pri_sh.stopDbCluster()
        self.log.info(result)
        result = self.pri_sh.startDbCluster()
        self.log.info(result)

        time.sleep(5)
        self.log.info('--------删除表-------')
        sql = f"drop table if exists {self.tb_name};"
        result = self.pri_sh.executDbSql(sql)
        self.log.info(result)
        self.log.info('---Opengauss_Reliability_Dbsys_Case245.py 执行结束-------')